package com.example.project.utils;

import java.util.*;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

/**
 * 高级NLP文本处理工具类
 */
public class AdvancedNLPUtil {
    
    // 中文停用词列表
    private static final Set<String> CHINESE_STOP_WORDS = new HashSet<>(Arrays.asList(
        "的", "了", "在", "是", "我", "有", "和", "就", "不", "人", "都", "一", "一个", "上", "也", "很", "到", "说", "要", "去", "你", "会", "着", "没有", "看", "好", "自己", "这", "那", "他", "她", "它", "我们", "你们", "他们", "这个", "那个", "这些", "那些", "什么", "怎么", "为什么", "因为", "所以", "但是", "而且", "或者", "如果", "虽然", "尽管", "为了", "通过", "关于", "对于", "根据", "按照", "由于", "自从", "直到", "除了", "除了……之外", "以及", "与其", "不如", "不管", "无论", "即使", "要是", "假如", "假使", "倘若", "要是", "若是", "若非", "只要", "只有", "除非", "无论", "不论", "不管", "任凭", "纵然", "哪怕", "就是", "即使", "即便", "即令", "就算", "纵使", "纵然", "倘使", "如果", "若", "如", "假设", "假若", "要是", "若是", "倘或", "倘使", "倘然", "如若", "果然", "万一", "的话", "那么", "那末", "然后", "于是", "接着", "跟着", "从而", "进而", "因而", "因此", "所以", "以致", "致使", "以便", "以免", "省得", "为了", "为着", "因为", "由于", "既然", "既", "固然", "虽然", "尽管", "纵然", "哪怕", "就是", "即使", "即便", "即令", "就算", "纵使", "纵然", "倘使", "如果", "若", "如", "假设", "假若", "要是", "若是", "倘或", "倘使", "倘然", "如若", "果然", "万一", "的话", "那么", "那末", "然后", "于是", "接着", "跟着", "从而", "进而", "因而", "因此", "所以", "以致", "致使", "以便", "以免", "省得", "为了", "为着", "因为", "由于", "既然", "既", "固然", "虽然", "尽管", "纵然", "哪怕", "就是", "即使", "即便", "即令", "就算", "纵使", "纵然", "倘使", "如果", "若", "如", "假设", "假若", "要是", "若是", "倘或", "倘使", "倘然", "如若", "果然", "万一"
    ));
    
    // 英文停用词列表
    private static final Set<String> ENGLISH_STOP_WORDS = new HashSet<>(Arrays.asList(
        "a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "by", "from", "up", "about", "into", "through", "during", "before", "after", "above", "below", "between", "among", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had", "do", "does", "did", "will", "would", "could", "should", "may", "might", "must", "can", "shall", "ought", "need", "dare", "used", "get", "got", "getting", "make", "made", "making", "take", "took", "taking", "come", "came", "coming", "go", "went", "going", "see", "saw", "seeing", "know", "knew", "knowing", "think", "thought", "thinking", "look", "looked", "looking", "want", "wanted", "wanting", "give", "gave", "giving", "tell", "told", "telling", "work", "worked", "working", "call", "called", "calling", "try", "tried", "trying", "ask", "asked", "asking", "need", "needed", "needing", "feel", "felt", "feeling", "become", "became", "becoming", "leave", "left", "leaving", "put", "put", "putting", "mean", "meant", "meaning", "keep", "kept", "keeping", "let", "let", "letting", "begin", "began", "beginning", "seem", "seemed", "seeming", "help", "helped", "helping", "talk", "talked", "talking", "turn", "turned", "turning", "start", "started", "starting", "show", "showed", "showing", "hear", "heard", "hearing", "play", "played", "playing", "run", "ran", "running", "move", "moved", "moving", "like", "liked", "liking", "live", "lived", "living", "believe", "believed", "believing", "hold", "held", "holding", "bring", "brought", "bringing", "happen", "happened", "happening", "write", "wrote", "writing", "provide", "provided", "providing", "sit", "sat", "sitting", "stand", "stood", "standing", "lose", "lost", "losing", "pay", "paid", "paying", "meet", "met", "meeting", "include", "included", "including", "continue", "continued", "continuing", "set", "set", "setting", "learn", "learned", "learning", "change", "changed", "changing", "lead", "led", "leading", "understand", "understood", "understanding", "watch", "watched", "watching", "follow", "followed", "following", "stop", "stopped", "stopping", "create", "created", "creating", "speak", "spoke", "speaking", "read", "read", "reading", "allow", "allowed", "allowing", "add", "added", "adding", "spend", "spent", "spending", "grow", "grew", "growing", "open", "opened", "opening", "walk", "walked", "walking", "win", "won", "winning", "offer", "offered", "offering", "remember", "remembered", "remembering", "love", "loved", "loving", "consider", "considered", "considering", "appear", "appeared", "appearing", "buy", "bought", "buying", "wait", "waited", "waiting", "serve", "served", "serving", "die", "died", "dying", "send", "sent", "sending", "build", "built", "building", "stay", "stayed", "staying", "fall", "fell", "falling", "cut", "cut", "cutting", "reach", "reached", "reaching", "kill", "killed", "killing", "remain", "remained", "remaining", "suggest", "suggested", "suggesting", "raise", "raised", "raising", "pass", "passed", "passing", "sell", "sold", "selling", "require", "required", "requiring", "report", "reported", "reporting", "decide", "decided", "deciding", "pull", "pulled", "pulling", "return", "returned", "returning", "explain", "explained", "explaining", "hope", "hoped", "hoping", "develop", "developed", "developing", "carry", "carried", "carrying", "break", "broke", "breaking", "receive", "received", "receiving", "agree", "agreed", "agreeing", "support", "supported", "supporting", "hit", "hit", "hitting", "produce", "produced", "producing", "eat", "ate", "eating", "cover", "covered", "covering", "catch", "caught", "catching", "draw", "drew", "drawing", "choose", "chose", "choosing", "wear", "wore", "wearing", "prepare", "prepared", "preparing", "accept", "accepted", "accepting", "sleep", "slept", "sleeping", "join", "joined", "joining", "drive", "drove", "driving", "worry", "worried", "worrying", "realize", "realized", "realizing", "bring", "brought", "bringing", "treat", "treated", "treating", "identify", "identified", "identifying", "throw", "threw", "throwing", "express", "expressed", "expressing", "apply", "applied", "applying", "argue", "argued", "arguing", "establish", "established", "establishing", "visit", "visited", "visiting", "avoid", "avoided", "avoiding", "imagine", "imagined", "imagining", "finish", "finished", "finishing", "wonder", "wondered", "wondering", "suppose", "supposed", "supposing", "recognize", "recognized", "recognizing", "manage", "managed", "managing", "affect", "affected", "affecting", "depend", "depended", "depending", "relate", "related", "relating", "discover", "discovered", "discovering", "notice", "noticed", "noticing", "complete", "completed", "completing", "refer", "referred", "referring", "sound", "sounded", "sounding", "improve", "improved", "improving", "contain", "contained", "containing", "prevent", "prevented", "preventing", "mention", "mentioned", "mentioning", "admit", "admitted", "admitting", "assume", "assumed", "assuming", "engage", "engaged", "engaging", "claim", "claimed", "claiming", "deserve", "deserved", "deserving", "reflect", "reflected", "reflecting", "encourage", "encouraged", "encouraging", "appreciate", "appreciated", "appreciating", "achieve", "achieved", "achieving", "select", "selected", "selecting", "experience", "experienced", "experiencing", "wish", "wished", "wishing", "train", "trained", "training", "warn", "warned", "warning", "celebrate", "celebrated", "celebrating", "promise", "promised", "promising", "introduce", "introduced", "introducing", "respond", "responded", "responding", "maintain", "maintained", "maintaining", "justify", "justified", "justifying", "describe", "described", "describing", "attempt", "attempted", "attempting", "prefer", "preferred", "preferring", "investigate", "investigated", "investigating", "obtain", "obtained", "obtaining", "ignore", "ignored", "ignoring", "share", "shared", "sharing", "confirm", "confirmed", "confirming", "solve", "solved", "solving", "tend", "tended", "tending", "concentrate", "concentrated", "concentrating", "discuss", "discussed", "discussing", "state", "stated", "stating", "reveal", "revealed", "revealing", "enable", "enabled", "enabling", "ensure", "ensured", "ensuring", "involve", "involved", "involving", "prove", "proved", "proving", "arise", "arose", "arising", "assess", "assessed", "assessing", "benefit", "benefited", "benefiting", "consist", "consisted", "consisting", "contribute", "contributed", "contributing", "demonstrate", "demonstrated", "demonstrating", "emerge", "emerged", "emerging", "extend", "extended", "extending", "imply", "implied", "implying", "indicate", "indicated", "indicating", "occur", "occurred", "occurring", "publish", "published", "publishing", "require", "required", "requiring", "seek", "sought", "seeking", "tend", "tended", "tending", "transform", "transformed", "transforming", "vary", "varied", "varying", "account", "accounted", "accounting", "address", "addressed", "addressing", "advance", "advanced", "advancing", "aid", "aided", "aiding", "alter", "altered", "altering", "analyze", "analyzed", "analyzing", "announce", "announced", "announcing", "appoint", "appointed", "appointing", "approve", "approved", "approving", "arrange", "arranged", "arranging", "assemble", "assembled", "assembling", "assign", "assigned", "assigning", "attach", "attached", "attaching", "authorize", "authorized", "authorizing", "calculate", "calculated", "calculating", "cancel", "canceled", "canceling", "capture", "captured", "capturing", "categorize", "categorized", "categorizing", "challenge", "challenged", "challenging", "characterize", "characterized", "characterizing", "circulate", "circulated", "circulating", "clarify", "clarified", "clarifying", "classify", "classified", "classifying", "combine", "combined", "combining", "commence", "commenced", "commencing", "comment", "commented", "commenting", "communicate", "communicated", "communicating", "compare", "compared", "comparing", "compete", "competed", "competing", "compile", "compiled", "compiling", "comprise", "comprised", "comprising", "conceive", "conceived", "conceiving", "conclude", "concluded", "concluding", "conduct", "conducted", "conducting", "confine", "confined", "confining", "confront", "confronted", "confronting", "connect", "connected", "connecting", "conserve", "conserved", "conserving", "consider", "considered", "considering", "consume", "consumed", "consuming", "contact", "contacted", "contacting", "contemplate", "contemplated", "contemplating", "contract", "contracted", "contracting", "contrast", "contrasted", "contrasting", "control", "controlled", "controlling", "convert", "converted", "converting", "cooperate", "cooperated", "cooperating", "coordinate", "coordinated", "coordinating", "correspond", "corresponded", "corresponding", "criticize", "criticized", "criticizing", "declare", "declared", "declaring", "decline", "declined", "declining", "dedicate", "dedicated", "dedicating", "defend", "defended", "defending", "define", "defined", "defining", "delay", "delayed", "delaying", "delete", "deleted", "deleting", "deliver", "delivered", "delivering", "demand", "demanded", "demanding", "deny", "denied", "denying", "depict", "depicted", "depicting", "derive", "derived", "deriving", "design", "designed", "designing", "detect", "detected", "detecting", "determine", "determined", "determining", "devise", "devised", "devising", "dictate", "dictated", "dictating", "differentiate", "differentiated", "differentiating", "diminish", "diminished", "diminishing", "direct", "directed", "directing", "disclose", "disclosed", "disclosing", "discount", "discounted", "discounting", "discourage", "discouraged", "discouraging", "dispute", "disputed", "disputing", "distinguish", "distinguished", "distinguishing", "distribute", "distributed", "distributing", "divert", "diverted", "diverting", "document", "documented", "documenting", "dominate", "dominated", "dominating", "donate", "donated", "donating", "draft", "drafted", "drafting", "edit", "edited", "editing", "educate", "educated", "educating", "elevate", "elevated", "elevating", "eliminate", "eliminated", "eliminating", "emphasize", "emphasized", "emphasizing", "employ", "employed", "employing", "empower", "empowered", "empowering", "enable", "enabled", "enabling", "encounter", "encountered", "encountering", "endorse", "endorsed", "endorsing", "enforce", "enforced", "enforcing", "enhance", "enhanced", "enhancing", "enrich", "enriched", "enriching", "enroll", "enrolled", "enrolling", "entail", "entailed", "entailing", "entertain", "entertained", "entertaining", "envision", "envisioned", "envisioning", "equip", "equipped", "equipping", "escalate", "escalated", "escalating", "estimate", "estimated", "estimating", "evaluate", "evaluated", "evaluating", "evolve", "evolved", "evolving", "examine", "examined", "examining", "exceed", "exceeded", "exceeding", "exclude", "excluded", "excluding", "execute", "executed", "executing", "exemplify", "exemplified", "exemplifying", "exhibit", "exhibited", "exhibiting", "expand", "expanded", "expanding", "expedite", "expedited", "expediting", "experiment", "experimented", "experimenting", "exploit", "exploited", "exploiting", "explore", "explored", "exploring", "export", "exported", "exporting", "expose", "exposed", "exposing", "extract", "extracted", "extracting", "facilitate", "facilitated", "facilitating", "favor", "favored", "favoring", "feature", "featured", "featuring", "file", "filed", "filing", "finance", "financed", "financing", "focus", "focused", "focusing", "forecast", "forecasted", "forecasting", "formulate", "formulated", "formulating", "foster", "fostered", "fostering", "frame", "framed", "framing", "fulfill", "fulfilled", "fulfilling", "function", "functioned", "functioning", "fund", "funded", "funding", "furnish", "furnished", "furnishing", "gain", "gained", "gaining", "generate", "generated", "generating", "govern", "governed", "governing", "grant", "granted", "granting", "guarantee", "guaranteed", "guaranteeing", "guide", "guided", "guiding", "handle", "handled", "handling", "harmonize", "harmonized", "harmonizing", "head", "headed", "heading", "heighten", "heightened", "heightening", "highlight", "highlighted", "highlighting", "hire", "hired", "hiring", "host", "hosted", "hosting", "hypothesize", "hypothesized", "hypothesizing", "identify", "identified", "identifying", "illuminate", "illuminated", "illuminating", "illustrate", "illustrated", "illustrating", "implement", "implemented", "implementing", "impose", "imposed", "imposing", "improve", "improved", "improving", "incorporate", "incorporated", "incorporating", "index", "indexed", "indexing", "induce", "induced", "inducing", "influence", "influenced", "influencing", "inform", "informed", "informing", "initiate", "initiated", "initiating", "innovate", "innovated", "innovating", "input", "inputted", "inputting", "inspect", "inspected", "inspecting", "inspire", "inspired", "inspiring", "install", "installed", "installing", "institute", "instituted", "instituting", "instruct", "instructed", "instructing", "insure", "insured", "insuring", "integrate", "integrated", "integrating", "intensify", "intensified", "intensifying", "interact", "interacted", "interacting", "interface", "interfaced", "interfacing", "interpret", "interpreted", "interpreting", "intervene", "intervened", "intervening", "interview", "interviewed", "interviewing", "introduce", "introduced", "introducing", "invent", "invented", "inventing", "invest", "invested", "investing", "invite", "invited", "inviting", "involve", "involved", "involving", "isolate", "isolated", "isolating", "issue", "issued", "issuing", "itemize", "itemized", "itemizing", "join", "joined", "joining", "justify", "justified", "justifying", "label", "labeled", "labeling", "launch", "launched", "launching", "lay", "laid", "laying", "lead", "led", "leading", "learn", "learned", "learning", "lease", "leased", "leasing", "lecture", "lectured", "lecturing", "legislate", "legislated", "legislating", "lend", "lent", "lending", "license", "licensed", "licensing", "link", "linked", "linking", "list", "listed", "listing", "locate", "located", "locating", "log", "logged", "logging", "lose", "lost", "losing", "maintain", "maintained", "maintaining", "make", "made", "making", "manage", "managed", "managing", "manipulate", "manipulated", "manipulating", "manufacture", "manufactured", "manufacturing", "map", "mapped", "mapping", "market", "marketed", "marketing", "master", "mastered", "mastering", "match", "matched", "matching", "maximize", "maximized", "maximizing", "measure", "measured", "measuring", "mediate", "mediated", "mediating", "meet", "met", "meeting", "merge", "merged", "merging", "migrate", "migrated", "migrating", "minimize", "minimized", "minimizing", "mobilize", "mobilized", "mobilizing", "model", "modeled", "modeling", "moderate", "moderated", "moderating", "modify", "modified", "modifying", "monitor", "monitored", "monitoring", "motivate", "motivated", "motivating", "mount", "mounted", "mounting", "move", "moved", "moving", "multiply", "multiplied", "multiplying", "name", "named", "naming", "navigate", "navigated", "navigating", "negotiate", "negotiated", "negotiating", "nominate", "nominated", "nominating", "normalize", "normalized", "normalizing", "notify", "notified", "notifying", "nurture", "nurtured", "nurturing", "observe", "observed", "observing", "obtain", "obtained", "obtaining", "occupy", "occupied", "occupying", "offset", "offset", "offsetting", "operate", "operated", "operating", "optimize", "optimized", "optimizing", "order", "ordered", "ordering", "organize", "organized", "organizing", "orient", "oriented", "orienting", "originate", "originated", "originating", "outline", "outlined", "outlining", "overcome", "overcame", "overcoming", "oversee", "oversaw", "overseeing", "owe", "owed", "owing", "own", "owned", "owning", "package", "packaged", "packaging", "paint", "painted", "painting", "panel", "paneled", "paneling", "parallel", "paralleled", "paralleling", "participate", "participated", "participating", "partner", "partnered", "partnering", "pass", "passed", "passing", "patent", "patented", "patenting", "pattern", "patterned", "patterning", "pause", "paused", "pausing", "pay", "paid", "paying", "peak", "peaked", "peaking", "penalize", "penalized", "penalizing", "perceive", "perceived", "perceiving", "perform", "performed", "performing", "permit", "permitted", "permitting", "persist", "persisted", "persisting", "personalize", "personalized", "personalizing", "persuade", "persuaded", "persuading", "phase", "phased", "phasing", "phone", "phoned", "phoning", "photograph", "photographed", "photographing", "pick", "picked", "picking", "pilot", "piloted", "piloting", "pinpoint", "pinpointed", "pinpointing", "pioneer", "pioneered", "pioneering", "place", "placed", "placing", "plan", "planned", "planning", "plant", "planted", "planting", "play", "played", "playing", "pledge", "pledged", "pledging", "plot", "plotted", "plotting", "point", "pointed", "pointing", "polish", "polished", "polishing", "poll", "polled", "polling", "pool", "pooled", "pooling", "populate", "populated", "populating", "portray", "portrayed", "portraying", "pose", "posed", "posing", "position", "positioned", "positioning", "possess", "possessed", "possessing", "post", "posted", "posting", "postpone", "postponed", "postponing", "power", "powered", "powering", "practice", "practiced", "practicing", "praise", "praised", "praising", "precede", "preceded", "preceding", "predict", "predicted", "predicting", "prefer", "preferred", "preferring", "prepare", "prepared", "preparing", "prescribe", "prescribed", "prescribing", "present", "presented", "presenting", "preserve", "preserved", "preserving", "preside", "presided", "presiding", "press", "pressed", "pressing", "pressure", "pressured", "pressuring", "presume", "presumed", "presuming", "prevent", "prevented", "preventing", "price", "priced", "pricing", "print", "printed", "printing", "prioritize", "prioritized", "prioritizing", "probe", "probed", "probing", "proceed", "proceeded", "proceeding", "process", "processed", "processing", "procure", "procured", "procuring", "produce", "produced", "producing", "profile", "profiled", "profiling", "profit", "profited", "profiting", "program", "programmed", "programming", "progress", "progressed", "progressing", "project", "projected", "projecting", "promise", "promised", "promising", "promote", "promoted", "promoting", "prompt", "prompted", "prompting", "proof", "proofed", "proofing", "propose", "proposed", "proposing", "protect", "protected", "protecting", "protest", "protested", "protesting", "prove", "proved", "proving", "provide", "provided", "providing", "provoke", "provoked", "provoking", "publicize", "publicized", "publicizing", "publish", "published", "publishing", "pull", "pulled", "pulling", "pump", "pumped", "pumping", "purchase", "purchased", "purchasing", "pursue", "pursued", "pursuing", "push", "pushed", "pushing", "qualify", "qualified", "qualifying", "quantify", "quantified", "quantifying", "question", "questioned", "questioning", "quote", "quoted", "quoting", "race", "raced", "racing", "raise", "raised", "raising", "rally", "rallied", "rallying", "randomize", "randomized", "randomizing", "range", "ranged", "ranging", "rank", "ranked", "ranking", "rate", "rated", "rating", "ratify", "ratified", "ratifying", "rationalize", "rationalized", "rationalizing", "reach", "reached", "reaching", "react", "reacted", "reacting", "read", "read", "reading", "realign", "realigned", "realigning", "realize", "realized", "realizing", "reallocate", "reallocated", "reallocating", "reason", "reasoned", "reasoning", "reassess", "reassessed", "reassessing", "reassign", "reassigned", "reassigning", "reassure", "reassured", "reassuring", "rebuild", "rebuilt", "rebuilding", "recall", "recalled", "recalling", "receive", "received", "receiving", "reconcile", "reconciled", "reconciling", "reconfigure", "reconfigured", "reconfiguring", "record", "recorded", "recording", "recover", "recovered", "recovering", "recruit", "recruited", "recruiting", "recycle", "recycled", "recycling", "redirect", "redirected", "redirecting", "reduce", "reduced", "reducing", "refer", "referred", "referring", "refine", "refined", "refining", "reflect", "reflected", "reflecting", "reform", "reformed", "reforming", "refrain", "refrained", "refraining", "refresh", "refreshed", "refreshing", "refund", "refunded", "refunding", "refuse", "refused", "refusing", "regard", "regarded", "regarding", "register", "registered", "registering", "regret", "regretted", "regretting", "regulate", "regulated", "regulating", "rehabilitate", "rehabilitated", "rehabilitating", "reinforce", "reinforced", "reinforcing", "reject", "rejected", "rejecting", "relate", "related", "relating", "relax", "relaxed", "relaxing", "release", "released", "releasing", "relieve", "relieved", "relieving", "relocate", "relocated", "relocating", "rely", "relied", "relying", "remain", "remained", "remaining", "remark", "remarked", "remarking", "remember", "remembered", "remembering", "remind", "reminded", "reminding", "remove", "removed", "removing", "render", "rendered", "rendering", "renew", "renewed", "renewing", "renovate", "renovated", "renovating", "rent", "rented", "renting", "reopen", "reopened", "reopening", "reorganize", "reorganized", "reorganizing", "repair", "repaired", "repairing", "repeat", "repeated", "repeating", "replace", "replaced", "replacing", "reply", "replied", "replying", "report", "reported", "reporting", "represent", "represented", "representing", "reproduce", "reproduced", "reproducing", "request", "requested", "requesting", "require", "required", "requiring", "rescue", "rescued", "rescuing", "research", "researched", "researching", "resemble", "resembled", "resembling", "reserve", "reserved", "reserving", "reset", "reset", "resetting", "reside", "resided", "residing", "resign", "resigned", "resigning", "resist", "resisted", "resisting", "resolve", "resolved", "resolving", "resort", "resorted", "resorting", "resource", "resourced", "resourcing", "respect", "respected", "respecting", "respond", "responded", "responding", "rest", "rested", "resting", "restore", "restored", "restoring", "restrain", "restrained", "restraining", "restrict", "restricted", "restricting", "restructure", "restructured", "restructuring", "result", "resulted", "resulting", "resume", "resumed", "resuming", "retain", "retained", "retaining", "retire", "retired", "retiring", "retreat", "retreated", "retreating", "retrieve", "retrieved", "retrieving", "return", "returned", "returning", "reveal", "revealed", "revealing", "review", "reviewed", "reviewing", "revise", "revised", "revising", "revive", "revived", "reviving", "revoke", "revoked", "revoking", "reward", "rewarded", "rewarding", "rid", "rid", "ridding", "ride", "rode", "riding", "ring", "rang", "ringing", "rise", "rose", "rising", "risk", "risked", "risking", "rival", "rivaled", "rivaling", "roll", "rolled", "rolling", "rotate", "rotated", "rotating", "round", "rounded", "rounding", "route", "routed", "routing", "rub", "rubbed", "rubbing", "rule", "ruled", "ruling", "run", "ran", "running", "rush", "rushed", "rushing", "sacrifice", "sacrificed", "sacrificing", "safeguard", "safeguarded", "safeguarding", "sail", "sailed", "sailing", "sample", "sampled", "sampling", "sanction", "sanctioned", "sanctioning", "satisfy", "satisfied", "satisfying", "save", "saved", "saving", "say", "said", "saying", "scale", "scaled", "scaling", "scan", "scanned", "scanning", "schedule", "scheduled", "scheduling", "scheme", "schemed", "scheming", "score", "scored", "scoring", "scramble", "scrambled", "scrambling", "screen", "screened", "screening", "scrutinize", "scrutinized", "scrutinizing", "seal", "sealed", "sealing", "search", "searched", "searching", "season", "seasoned", "seasoning", "seat", "seated", "seating", "secure", "secured", "securing", "see", "saw", "seeing", "seek", "sought", "seeking", "seem", "seemed", "seeming", "select", "selected", "selecting", "sell", "sold", "selling", "send", "sent", "sending", "sense", "sensed", "sensing", "separate", "separated", "separating", "sequence", "sequenced", "sequencing", "serve", "served", "serving", "service", "serviced", "servicing", "set", "set", "setting", "settle", "settled", "settling", "shape", "shaped", "shaping", "share", "shared", "sharing", "shed", "shed", "shedding", "shell", "shelled", "shelling", "shelter", "sheltered", "sheltering", "shift", "shifted", "shifting", "ship", "shipped", "shipping", "shoot", "shot", "shooting", "shop", "shopped", "shopping", "shorten", "shortened", "shortening", "shoulder", "shouldered", "shouldering", "show", "showed", "showing", "shrink", "shrank", "shrinking", "shut", "shut", "shutting", "sight", "sighted", "sighting", "sign", "signed", "signing", "signal", "signaled", "signaling", "simplify", "simplified", "simplifying", "simulate", "simulated", "simulating", "sing", "sang", "singing", "sink", "sank", "sinking", "sit", "sat", "sitting", "size", "sized", "sizing", "sketch", "sketched", "sketching", "skip", "skipped", "skipping", "slide", "slid", "sliding", "slip", "slipped", "slipping", "slow", "slowed", "slowing", "smile", "smiled", "smiling", "smooth", "smoothed", "smoothing", "snap", "snapped", "snapping", "solve", "solved", "solving", "sort", "sorted", "sorting", "sound", "sounded", "sounding", "source", "sourced", "sourcing", "space", "spaced", "spacing", "span", "spanned", "spanning", "spare", "spared", "sparing", "spark", "sparked", "sparking", "speak", "spoke", "speaking", "specify", "specified", "specifying", "speculate", "speculated", "speculating", "speed", "sped", "speeding", "spell", "spelled", "spelling", "spend", "spent", "spending", "spin", "spun", "spinning", "split", "split", "splitting", "sponsor", "sponsored", "sponsoring", "spot", "spotted", "spotting", "spread", "spread", "spreading", "spring", "sprang", "springing", "spur", "spurred", "spurring", "squeeze", "squeezed", "squeezing", "stabilize", "stabilized", "stabilizing", "staff", "staffed", "staffing", "stage", "staged", "staging", "stain", "stained", "staining", "stake", "staked", "staking", "stamp", "stamped", "stamping", "stand", "stood", "standing", "standardize", "standardized", "standardizing", "start", "started", "starting", "state", "stated", "stating", "station", "stationed", "stationing", "stay", "stayed", "staying", "steer", "steered", "steering", "step", "stepped", "stepping", "stick", "stuck", "sticking", "stimulate", "stimulated", "stimulating", "stipulate", "stipulated", "stipulating", "stock", "stocked", "stocking", "stop", "stopped", "stopping", "store", "stored", "storing", "storm", "stormed", "storming", "strain", "strained", "straining", "stream", "streamed", "streaming", "streamline", "streamlined", "streamlining", "strengthen", "strengthened", "strengthening", "stress", "stressed", "stressing", "stretch", "stretched", "stretching", "strike", "struck", "striking", "string", "strung", "stringing", "strip", "stripped", "stripping", "strive", "strove", "striving", "structure", "structured", "structuring", "study", "studied", "studying", "style", "styled", "styling", "subject", "subjected", "subjecting", "submit", "submitted", "submitting", "subordinate", "subordinated", "subordinating", "subscribe", "subscribed", "subscribing", "subsidize", "subsidized", "subsidizing", "substantiate", "substantiated", "substantiating", "substitute", "substituted", "substituting", "subtract", "subtracted", "subtracting", "succeed", "succeeded", "succeeding", "sue", "sued", "suing", "suffer", "suffered", "suffering", "suggest", "suggested", "suggesting", "suit", "suited", "suiting", "summarize", "summarized", "summarizing", "supervise", "supervised", "supervising", "supplement", "supplemented", "supplementing", "supply", "supplied", "supplying", "support", "supported", "supporting", "suppose", "supposed", "supposing", "suppress", "suppressed", "suppressing", "surface", "surfaced", "surfacing", "surge", "surged", "surging", "surpass", "surpassed", "surpassing", "surprise", "surprised", "surprising", "surrender", "surrendered", "surrendering", "surround", "surrounded", "surrounding", "survey", "surveyed", "surveying", "survive", "survived", "surviving", "suspect", "suspected", "suspecting", "suspend", "suspended", "suspending", "sustain", "sustained", "sustaining", "swallow", "swallowed", "swallowing", "swap", "swapped", "swapping", "swear", "swore", "swearing", "sweep", "swept", "sweeping", "swell", "swelled", "swelling", "swim", "swam", "swimming", "swing", "swung", "swinging", "switch", "switched", "switching", "symbolize", "symbolized", "symbolizing", "sympathize", "sympathized", "sympathizing", "synchronize", "synchronized", "synchronizing", "synthesize", "synthesized", "synthesizing", "systematize", "systematized", "systematizing", "tabulate", "tabulated", "tabulating", "tailor", "tailored", "tailoring", "take", "took", "taking", "talk", "talked", "talking", "tame", "tamed", "taming", "tap", "tapped", "tapping", "target", "targeted", "targeting", "task", "tasked", "tasking", "taste", "tasted", "tasting", "tax", "taxed", "taxing", "teach", "taught", "teaching", "team", "teamed", "teaming", "tear", "tore", "tearing", "tease", "teased", "teasing", "telephone", "telephoned", "telephoning", "tell", "told", "telling", "tempt", "tempted", "tempting", "tend", "tended", "tending", "tense", "tensed", "tensing", "term", "termed", "terming", "terminate", "terminated", "terminating", "test", "tested", "testing", "testify", "testified", "testifying", "thank", "thanked", "thanking", "theorize", "theorized", "theorizing", "thicken", "thickened", "thickening", "think", "thought", "thinking", "threaten", "threatened", "threatening", "thrive", "thrived", "thriving", "throw", "threw", "throwing", "thrust", "thrust", "thrusting", "tie", "tied", "tying", "tighten", "tightened", "tightening", "time", "timed", "timing", "tip", "tipped", "tipping", "tire", "tired", "tiring", "title", "titled", "titling", "tolerate", "tolerated", "tolerating", "top", "topped", "topping", "total", "totaled", "totaling", "touch", "touched", "touching", "tour", "toured", "touring", "trace", "traced", "tracing", "track", "tracked", "tracking", "trade", "traded", "trading", "trail", "trailed", "trailing", "train", "trained", "training", "transfer", "transferred", "transferring", "transform", "transformed", "transforming", "translate", "translated", "translating", "transmit", "transmitted", "transmitting", "transport", "transported", "transporting", "trap", "trapped", "trapping", "travel", "traveled", "traveling", "treat", "treated", "treating", "trend", "trended", "trending", "trigger", "triggered", "triggering", "trim", "trimmed", "trimming", "trip", "tripped", "tripping", "trouble", "troubled", "troubling", "trust", "trusted", "trusting", "try", "tried", "trying", "tune", "tuned", "tuning", "turn", "turned", "turning", "tutor", "tutored", "tutoring", "twist", "twisted", "twisting", "type", "typed", "typing", "undergo", "underwent", "undergoing", "understand", "understood", "understanding", "undertake", "undertook", "undertaking", "unify", "unified", "unifying", "unite", "united", "uniting", "unload", "unloaded", "unloading", "unlock", "unlocked", "unlocking", "update", "updated", "updating", "upgrade", "upgraded", "upgrading", "uphold", "upheld", "upholding", "upset", "upset", "upsetting", "urge", "urged", "urging", "use", "used", "using", "utilize", "utilized", "utilizing", "vacate", "vacated", "vacating", "validate", "validated", "validating", "value", "valued", "valuing", "vanish", "vanished", "vanishing", "vary", "varied", "varying", "verify", "verified", "verifying", "veto", "vetoed", "vetoing", "view", "viewed", "viewing", "violate", "violated", "violating", "visit", "visited", "visiting", "visualize", "visualized", "visualizing", "voice", "voiced", "voicing", "volunteer", "volunteered", "volunteering", "vote", "voted", "voting", "wait", "waited", "waiting", "wake", "woke", "waking", "walk", "walked", "walking", "wander", "wandered", "wandering", "want", "wanted", "wanting", "warm", "warmed", "warming", "warn", "warned", "warning", "wash", "washed", "washing", "waste", "wasted", "wasting", "watch", "watched", "watching", "water", "watered", "watering", "wave", "waved", "waving", "weaken", "weakened", "weakening", "wear", "wore", "wearing", "weave", "wove", "weaving", "wed", "wed", "wedding", "weigh", "weighed", "weighing", "welcome", "welcomed", "welcoming", "widen", "widened", "widening", "win", "won", "winning", "wind", "wound", "winding", "wipe", "wiped", "wiping", "wish", "wished", "wishing", "withdraw", "withdrew", "withdrawing", "withhold", "withheld", "withholding", "withstand", "withstood", "withstanding", "witness", "witnessed", "witnessing", "wonder", "wondered", "wondering", "work", "worked", "working", "worry", "worried", "worrying", "worship", "worshiped", "worshiping", "wrap", "wrapped", "wrapping", "wreck", "wrecked", "wrecking", "write", "wrote", "writing", "yield", "yielded", "yielding"
    ));
    
    // 中文文本匹配模式
    private static final Pattern CHINESE_PATTERN = Pattern.compile("[\\u4e00-\\u9fa5]+");
    
    /**
     * 高级关键词提取 - 支持中英文混合文本
     */
    public static List<String> extractKeywordsAdvanced(String text, int maxKeywords) {
        if (text == null || text.trim().isEmpty()) {
            return new ArrayList<>();
        }
        
        // 清理文本
        String cleanText = cleanText(text);
        
        // 提取中文关键词
        List<String> chineseKeywords = extractChineseKeywords(cleanText);
        
        // 提取英文关键词
        List<String> englishKeywords = extractEnglishKeywords(cleanText);
        
        // 合并结果并按重要性排序
        List<String> allKeywords = new ArrayList<>();
        allKeywords.addAll(chineseKeywords);
        allKeywords.addAll(englishKeywords);
        
        // 去重并限制数量
        return allKeywords.stream()
                .distinct()
                .limit(maxKeywords)
                .collect(Collectors.toList());
    }
    
    /**
     * 提取中文关键词
     */
    private static List<String> extractChineseKeywords(String text) {
        List<String> keywords = new ArrayList<>();
        
        // 使用正则表达式匹配中文字符
        java.util.regex.Matcher matcher = CHINESE_PATTERN.matcher(text);
        
        while (matcher.find()) {
            String word = matcher.group();
            
            // 过滤停用词和短词
            if (!CHINESE_STOP_WORDS.contains(word) && word.length() >= 2) {
                keywords.add(word);
            }
        }
        
        return keywords;
    }
    
    /**
     * 提取英文关键词
     */
    private static List<String> extractEnglishKeywords(String text) {
        Map<String, Integer> wordFreq = new HashMap<>();
        
        // 分割成单词
        String[] words = text.toLowerCase().split("[^a-zA-Z]+");
        
        for (String word : words) {
            if (word.length() > 2 && !ENGLISH_STOP_WORDS.contains(word)) {
                wordFreq.put(word, wordFreq.getOrDefault(word, 0) + 1);
            }
        }
        
        // 按词频排序
        return wordFreq.entrySet().stream()
                .sorted((a, b) -> b.getValue().compareTo(a.getValue()))
                .map(Map.Entry::getKey)
                .collect(Collectors.toList());
    }
    
    /**
     * 清理文本内容
     */
    private static String cleanText(String text) {
        // 移除HTML标签
        String clean = text.replaceAll("<[^>]*>", " ");
        
        // 移除特殊字符，保留中英文和数字
        clean = clean.replaceAll("[^a-zA-Z0-9\\u4e00-\\u9fa5\\s]", " ");
        
        // 合并多个空格
        clean = clean.replaceAll("\\s+", " ");
        
        return clean.trim();
    }
    
    /**
     * 计算文本相似度（改进版）
     */
    public static double calculateTextSimilarity(String text1, String text2) {
        if (text1 == null || text2 == null) {
            return 0.0;
        }
        
        // 提取关键词
        List<String> keywords1 = extractKeywordsAdvanced(text1, 20);
        List<String> keywords2 = extractKeywordsAdvanced(text2, 20);
        
        if (keywords1.isEmpty() && keywords2.isEmpty()) {
            return 1.0;
        }
        
        if (keywords1.isEmpty() || keywords2.isEmpty()) {
            return 0.0;
        }
        
        // 计算Jaccard相似度
        Set<String> set1 = new HashSet<>(keywords1);
        Set<String> set2 = new HashSet<>(keywords2);
        
        Set<String> intersection = new HashSet<>(set1);
        intersection.retainAll(set2);
        
        Set<String> union = new HashSet<>(set1);
        union.addAll(set2);
        
        return (double) intersection.size() / union.size();
    }
    
    /**
     * 文本分类（基于关键词匹配）
     */
    public static String classifyText(String text, Map<String, List<String>> categoryKeywords) {
        if (text == null || categoryKeywords == null || categoryKeywords.isEmpty()) {
            return "未知";
        }
        
        List<String> textKeywords = extractKeywordsAdvanced(text, 10);
        
        String bestCategory = "未知";
        double bestScore = 0.0;
        
        for (Map.Entry<String, List<String>> entry : categoryKeywords.entrySet()) {
            String category = entry.getKey();
            List<String> categoryKeys = entry.getValue();
            
            // 计算匹配度
            double score = calculateCategoryMatch(textKeywords, categoryKeys);
            
            if (score > bestScore) {
                bestScore = score;
                bestCategory = category;
            }
        }
        
        return bestScore > 0.3 ? bestCategory : "未知";
    }
    
    /**
     * 计算关键词与分类的匹配度
     */
    private static double calculateCategoryMatch(List<String> textKeywords, List<String> categoryKeywords) {
        Set<String> textSet = new HashSet<>(textKeywords);
        Set<String> categorySet = new HashSet<>(categoryKeywords);
        
        Set<String> intersection = new HashSet<>(textSet);
        intersection.retainAll(categorySet);
        
        return (double) intersection.size() / categorySet.size();
    }
    
    /**
     * 情感分析（基础版）
     */
    public static String analyzeSentiment(String text) {
        if (text == null || text.trim().isEmpty()) {
            return "中性";
        }
        
        // 简单的情感词词典
        Set<String> positiveWords = new HashSet<>(Arrays.asList(
            "好", "优秀", "完美", "满意", "成功", "顺利", "高兴", "开心", "喜欢", "爱",
            "great", "excellent", "perfect", "good", "happy", "love", "like", "success", "wonderful"
        ));
        
        Set<String> negativeWords = new HashSet<>(Arrays.asList(
            "差", "糟糕", "失败", "问题", "错误", "困难", "麻烦", "失望", "生气", "讨厌",
            "bad", "terrible", "awful", "problem", "error", "difficult", "trouble", "disappoint", "hate"
        ));
        
        List<String> keywords = extractKeywordsAdvanced(text, 20);
        
        int positiveCount = 0;
        int negativeCount = 0;
        
        for (String keyword : keywords) {
            if (positiveWords.contains(keyword)) {
                positiveCount++;
            } else if (negativeWords.contains(keyword)) {
                negativeCount++;
            }
        }
        
        if (positiveCount > negativeCount) {
            return "积极";
        } else if (negativeCount > positiveCount) {
            return "消极";
        } else {
            return "中性";
        }
    }
    
    /**
     * 文本摘要生成
     */
    public static String generateSummary(String text, int maxSentences) {
        if (text == null || text.trim().isEmpty()) {
            return "";
        }
        
        // 简单的句子分割
        String[] sentences = text.split("[。！？.!?]+");
        
        if (sentences.length <= maxSentences) {
            return String.join("。", sentences) + "。";
        }
        
        // 按句子长度排序（简单的重要性评估）
        List<String> sortedSentences = Arrays.stream(sentences)
                .filter(s -> s.trim().length() > 5)
                .sorted((a, b) -> Integer.compare(b.length(), a.length()))
                .limit(maxSentences)
                .collect(Collectors.toList());
        
        return String.join("。", sortedSentences) + "。";
    }
}